/*
 * $Id: TokenizerTests.m 249 2008-09-09 13:58:25Z jon $
 * Sleeper
 *
 * Created by Jon Gordon on 12/28/07
 * Copyright Jon Gordon.  All rights reserved.
 * 
 * Permission is hereby granted, free of charge, to any person obtaining 
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including 
 * without limitation the rights to use, copy, modify, merge, publish, 
 * distribute, sublicense, and/or sell copies of the Software, and to 
 * permit persons to whom the Software is furnished to do so, subject to 
 * the following conditions:
 * 
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */


#import "TokenizerTests.h"

#import "QETokenizer.h"


@implementation TokenizerTests

- (void) testTokenizer
{
    QETokenizer *tokenizer = [[QETokenizer alloc] init];
    STAssertNotNil(tokenizer, @"failed to created new default tokenizer");
    
    NSArray *tokens = [tokenizer tokenize:@""];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@" "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"a b cccccccc  d eeee 123549 arqirf99q ;'09m"];
    STAssertNotNil(tokens, @"expected array of 8 tokens but got nil instead");
    STAssertTrue(8 == [tokens count], @"token array has %d elements instead of expected 8", [tokens count]);
    STAssertEqualObjects([tokens objectAtIndex:0],
                         @"a",
                         @"token at index 0 was \"%@\" instead of expected \"a\"",
                         [tokens objectAtIndex:0]);
    STAssertEqualObjects([tokens objectAtIndex:1],
                         @"b",
                         @"token at index 1 was \"%@\" instead of expected \"b\"",
                         [tokens objectAtIndex:1]);
    STAssertEqualObjects([tokens objectAtIndex:2],
                         @"cccccccc",
                         @"token at index 2 was \"%@\" instead of expected \"cccccccc\"",
                         [tokens objectAtIndex:2]);
    STAssertEqualObjects([tokens objectAtIndex:3],
                         @"d",
                         @"token at index 3 was \"%@\" instead of expected \"d\"",
                         [tokens objectAtIndex:3]);
    STAssertEqualObjects([tokens objectAtIndex:4],
                         @"eeee",
                         @"token at index 4 was \"%@\" instead of expected \"eeee\"",
                         [tokens objectAtIndex:4]);
    STAssertEqualObjects([tokens objectAtIndex:5],
                         @"123549",
                         @"token at index 5 was \"%@\" instead of expected \"123549\"",
                         [tokens objectAtIndex:5]);
    STAssertEqualObjects([tokens objectAtIndex:6],
                         @"arqirf99q",
                         @"token at index 6 was \"%@\" instead of expected \"arqirf99q\"",
                         [tokens objectAtIndex:6]);
    STAssertEqualObjects([tokens objectAtIndex:7],
                         @";'09m",
                         @"token at index 7 was \"%@\" instead of expected \";'09m\"",
                         [tokens objectAtIndex:7]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"     "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    [tokenizer release];
    
    tokenizer = [[QETokenizer alloc] initWithDelimiter:' '];
    STAssertNotNil(tokenizer, @"failed to created new tokenizer with delimiter \" \"");
    
    tokens = [tokenizer tokenize:@""];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@" "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"a b cccccccc  d eeee 123549 arqirf99q ;'09m"];
    STAssertNotNil(tokens, @"expected array of 8 tokens but got nil instead");
    STAssertTrue(8 == [tokens count], @"token array has %d elements instead of expected 8", [tokens count]);
    STAssertEqualObjects([tokens objectAtIndex:0],
                         @"a",
                         @"token at index 0 was \"%@\" instead of expected \"a\"",
                         [tokens objectAtIndex:0]);
    STAssertEqualObjects([tokens objectAtIndex:1],
                         @"b",
                         @"token at index 1 was \"%@\" instead of expected \"b\"",
                         [tokens objectAtIndex:1]);
    STAssertEqualObjects([tokens objectAtIndex:2],
                         @"cccccccc",
                         @"token at index 2 was \"%@\" instead of expected \"cccccccc\"",
                         [tokens objectAtIndex:2]);
    STAssertEqualObjects([tokens objectAtIndex:3],
                         @"d",
                         @"token at index 3 was \"%@\" instead of expected \"d\"",
                         [tokens objectAtIndex:3]);
    STAssertEqualObjects([tokens objectAtIndex:4],
                         @"eeee",
                         @"token at index 4 was \"%@\" instead of expected \"eeee\"",
                         [tokens objectAtIndex:4]);
    STAssertEqualObjects([tokens objectAtIndex:5],
                         @"123549",
                         @"token at index 5 was \"%@\" instead of expected \"123549\"",
                         [tokens objectAtIndex:5]);
    STAssertEqualObjects([tokens objectAtIndex:6],
                         @"arqirf99q",
                         @"token at index 6 was \"%@\" instead of expected \"arqirf99q\"",
                         [tokens objectAtIndex:6]);
    STAssertEqualObjects([tokens objectAtIndex:7],
                         @";'09m",
                         @"token at index 7 was \"%@\" instead of expected \";'09m\"",
                         [tokens objectAtIndex:7]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"     "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    [tokenizer release];
    
    tokenizer = [[QETokenizer alloc] initWithDelimiter:'/'];
    STAssertNotNil(tokenizer, @"failed to created new tokenizer with single delimiter \"/\"");
    
    tokens = [tokenizer tokenize:@""];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"/"];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"//"];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"a/b/cccccc c//d/eeee/123549/arqirf99q/;'09m"];
    STAssertNotNil(tokens, @"expected array of 8 tokens but got nil instead");
    STAssertTrue(8 == [tokens count], @"token array has %d elements instead of expected 8", [tokens count]);
    STAssertEqualObjects([tokens objectAtIndex:0],
                         @"a",
                         @"token at index 0 was \"%@\" instead of expected \"a\"",
                         [tokens objectAtIndex:0]);
    STAssertEqualObjects([tokens objectAtIndex:1],
                         @"b",
                         @"token at index 1 was \"%@\" instead of expected \"b\"",
                         [tokens objectAtIndex:1]);
    STAssertEqualObjects([tokens objectAtIndex:2],
                         @"cccccc c",
                         @"token at index 2 was \"%@\" instead of expected \"cccccc c\"",
                         [tokens objectAtIndex:2]);
    STAssertEqualObjects([tokens objectAtIndex:3],
                         @"d",
                         @"token at index 3 was \"%@\" instead of expected \"d\"",
                         [tokens objectAtIndex:3]);
    STAssertEqualObjects([tokens objectAtIndex:4],
                         @"eeee",
                         @"token at index 4 was \"%@\" instead of expected \"eeee\"",
                         [tokens objectAtIndex:4]);
    STAssertEqualObjects([tokens objectAtIndex:5],
                         @"123549",
                         @"token at index 5 was \"%@\" instead of expected \"123549\"",
                         [tokens objectAtIndex:5]);
    STAssertEqualObjects([tokens objectAtIndex:6],
                         @"arqirf99q",
                         @"token at index 6 was \"%@\" instead of expected \"arqirf99q\"",
                         [tokens objectAtIndex:6]);
    STAssertEqualObjects([tokens objectAtIndex:7],
                         @";'09m",
                         @"token at index 7 was \"%@\" instead of expected \";'09m\"",
                         [tokens objectAtIndex:7]);
    
    tokens = [tokenizer tokenize:@"//"];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"/////"];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    [tokenizer release];
    
    tokenizer = [[QETokenizer alloc] initWithDelimitersFromString:@"/ "];
    STAssertNotNil(tokenizer, @"failed to created new tokenizer with delimiters \" /\"");
    
    tokens = [tokenizer tokenize:@""];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@" "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"/ "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"a b/cccccccc /d eeee 123549/ /arqirf99q ;'09m"];
    STAssertNotNil(tokens, @"expected array of 8 tokens but got nil instead");
    STAssertTrue(8 == [tokens count], @"token array has %d elements instead of expected 8", [tokens count]);
    STAssertEqualObjects([tokens objectAtIndex:0],
                         @"a",
                         @"token at index 0 was \"%@\" instead of expected \"a\"",
                         [tokens objectAtIndex:0]);
    STAssertEqualObjects([tokens objectAtIndex:1],
                         @"b",
                         @"token at index 1 was \"%@\" instead of expected \"b\"",
                         [tokens objectAtIndex:1]);
    STAssertEqualObjects([tokens objectAtIndex:2],
                         @"cccccccc",
                         @"token at index 2 was \"%@\" instead of expected \"cccccccc\"",
                         [tokens objectAtIndex:2]);
    STAssertEqualObjects([tokens objectAtIndex:3],
                         @"d",
                         @"token at index 3 was \"%@\" instead of expected \"d\"",
                         [tokens objectAtIndex:3]);
    STAssertEqualObjects([tokens objectAtIndex:4],
                         @"eeee",
                         @"token at index 4 was \"%@\" instead of expected \"eeee\"",
                         [tokens objectAtIndex:4]);
    STAssertEqualObjects([tokens objectAtIndex:5],
                         @"123549",
                         @"token at index 5 was \"%@\" instead of expected \"123549\"",
                         [tokens objectAtIndex:5]);
    STAssertEqualObjects([tokens objectAtIndex:6],
                         @"arqirf99q",
                         @"token at index 6 was \"%@\" instead of expected \"arqirf99q\"",
                         [tokens objectAtIndex:6]);
    STAssertEqualObjects([tokens objectAtIndex:7],
                         @";'09m",
                         @"token at index 7 was \"%@\" instead of expected \";'09m\"",
                         [tokens objectAtIndex:7]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@" /    //"];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    [tokenizer release];
    
    tokenizer = [[QETokenizer alloc] initWithDelimitersFromCharacterSet:[NSCharacterSet whitespaceCharacterSet]];
    STAssertNotNil(tokenizer, @"failed to created new tokenizer with delimiters from whitespaceCharacterSet");
    
    tokens = [tokenizer tokenize:@""];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@" "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"a b\tcccccccc  d eeee\t123549 arqirf99q ;'09m"];
    STAssertNotNil(tokens, @"expected array of 8 tokens but got nil instead");
    STAssertTrue(8 == [tokens count], @"token array has %d elements instead of expected 8", [tokens count]);
    STAssertEqualObjects([tokens objectAtIndex:0],
                         @"a",
                         @"token at index 0 was \"%@\" instead of expected \"a\"",
                         [tokens objectAtIndex:0]);
    STAssertEqualObjects([tokens objectAtIndex:1],
                         @"b",
                         @"token at index 1 was \"%@\" instead of expected \"b\"",
                         [tokens objectAtIndex:1]);
    STAssertEqualObjects([tokens objectAtIndex:2],
                         @"cccccccc",
                         @"token at index 2 was \"%@\" instead of expected \"cccccccc\"",
                         [tokens objectAtIndex:2]);
    STAssertEqualObjects([tokens objectAtIndex:3],
                         @"d",
                         @"token at index 3 was \"%@\" instead of expected \"d\"",
                         [tokens objectAtIndex:3]);
    STAssertEqualObjects([tokens objectAtIndex:4],
                         @"eeee",
                         @"token at index 4 was \"%@\" instead of expected \"eeee\"",
                         [tokens objectAtIndex:4]);
    STAssertEqualObjects([tokens objectAtIndex:5],
                         @"123549",
                         @"token at index 5 was \"%@\" instead of expected \"123549\"",
                         [tokens objectAtIndex:5]);
    STAssertEqualObjects([tokens objectAtIndex:6],
                         @"arqirf99q",
                         @"token at index 6 was \"%@\" instead of expected \"arqirf99q\"",
                         [tokens objectAtIndex:6]);
    STAssertEqualObjects([tokens objectAtIndex:7],
                         @";'09m",
                         @"token at index 7 was \"%@\" instead of expected \";'09m\"",
                         [tokens objectAtIndex:7]);
    
    tokens = [tokenizer tokenize:@"  "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    tokens = [tokenizer tokenize:@"     "];
    STAssertNotNil(tokens, @"expected zero-length token array but got nil instead");
    STAssertTrue(0 == [tokens count], @"token array has %d elements instead of expected zero", [tokens count]);
    
    [tokenizer release];
    
} // testTokenizer

@end
